import logging,os,gc,sys

from basic import check2skip,CleanText,Parse,W2vector
from optparse import OptionParser

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\
    level=logging.INFO)


def preprocess(options,filename):
    #parse chinese documents
    parse_file = "%s_parse"%filename
    parse = Parse(filename,options.parse_dict)
    parse.parse_text(parse_file)
    #clean punctuation and stopwords
    clean = CleanText(options.clean_stopwords,options.clean_punctuation)
    sentences = []
    for i,words in enumerate(open(parse_file)):
        words = words.strip().split('/')
        clean_words = clean.clean_punc(words)
        clean_words = clean.clean_stopwords(clean_words)
        sentences.append(clean_words)
        if i%1000==0:
            print i
       
    print "del ..."
    del clean
    gc.collect()
    return sentences
    
def train_w2v(options,filename):
    name = filename.split('/')[-1]
    model_file = "model/%s.word2vec"%name
    if check2skip(model_file,options.overwrite):
	print "check: %s exists ... skip"%model_file
   	exit(0)
    sentences = preprocess(options,filename)
    w2v = W2vector(options.feat_dim,options.min_word_count,options.num_workers,options.context,options.downsampling)
    w2v.train(sentences)
    w2v.save_model(model_file)

def main(argv=None):

    if argv is None:
        argv = sys.argv[1:]

    parser = OptionParser(usage="""usage: %prog [options] text_file_path """)

    parser.add_option("--overwrite", default=0, type="int", help="overwrite existing w2v model file (default: 0)")

    parser.add_option('--parse_dict', type="string", default='', help='The user dictionary for jieba parse, if not give, system will use default dictionary of jieba program (default: None)')
    parser.add_option('--clean_stopwords', type="string", default='stopwords.txt', help='The stopwords sets for clean (default:stopwords.txt)')
    parser.add_option('--clean_punctuation', type="string", default='punctuation.txt', help='The punctuation sets for clean (default:punctuation.txt)')

    parser.add_option('--feat_dim', type="int", default=300, help='The vector dimension of word2vector training (default:300)')
    parser.add_option('--min_word_count', type="int", default=40, help='The min_word_count of word2vector training (default:40)')
    parser.add_option('--num_workers', type="int", default=4, help='The num_workers of word2vector training (default:4)')
    parser.add_option('--context', type="int", default=10, help='The context of word2vector training (default:10)')
    parser.add_option('--downsampling', type="float", default=1e-3, help='The downsampling of word2vector training (default:1e-3)')

    (options, args) = parser.parse_args(argv)
    if len(args) < 1:
        parser.print_help()
        return 1
    
    return train_w2v(options, args[0])

if __name__ == '__main__':
    sys.exit(main())
    





